*********************************************************
*** This do.file prepares Panel A of Figure 2
*********************************************************

global db "D:\Dropbox\unequal_gains\main_data"
global resultspath "D:\Dropbox\unequal_gains\QJE revision plan\analysis\clean_results"
global Appendixresultspath "D:\Dropbox\unequal_gains\QJE revision plan\analysis\clean_results\appendix"

** Panel A(a) **

** 1. Compute spending on new & about-to-exit goods (that didn't exist at any point in the previous year
** or in any of the following years, using both the HMS and RMS datasets)
clear
set obs 1 
generate year=.
save "$db/Important Price Datasets/new_exit_income_final.dta", replace

* generate files for 2003 and 2016 so that we can run one loop, then we re-arrange the files below
clear
set obs 1
generate upc=.
generate upc_ver_uc=.
save "$db/Important Lists/available_upcs_upto2003_final.dta", replace
save "$db/Important Lists/available_upcs_post2015_final.dta", replace

foreach i of numlist 2004(1)2015 {

local prev=`i'-1

use "$db/Important Price Datasets/price_index_household_income_`i'_2004_modules_final.dta", clear

merge m:1 upc upc_ver using "$db/Important Lists/available_upcs_upto`prev'_final.dta"
drop if _merge==2
gen new=(_merge==1)
drop _merge

merge m:1 upc upc_ver using "$db/Important Lists/available_upcs_post`i'_final.dta"
drop if _merge==2
gen exit=(_merge==1)

drop total_spending*

* compute overall spending shares on new/old by quintiles: 
bysort income_quintile: egen double total_spending=sum(average_unit_price*total_quantity)
bysort income_quintile: egen double total_spending_new=sum(average_unit_price*total_quantity*new)
bysort income_quintile: egen double total_spending_exit=sum(average_unit_price*total_quantity*exit)

gen ssnp  = total_spending_new/total_spending
gen ssep  = total_spending_exit/total_spending

* compute overall spending shares on new/old by quintiles & product modules:
bysort income_quintile product_module: egen double total_spendingM=sum(average_unit_price*total_quantity)
bysort income_quintile product_module: egen double total_spendingM_new=sum(average_unit_price*total_quantity*new)
bysort income_quintile product_module: egen double total_spendingM_exit=sum(average_unit_price*total_quantity*exit)

gen ssnpM  = total_spendingM_new/total_spendingM
gen ssepM  = total_spendingM_exit/total_spendingM

keep income_q product_module total_spending* ssnp* ssep* 
duplicates drop 

gen year=`i'
append using "$db/Important Price Datasets/new_exit_income_final.dta"
drop if missing(year)
save "$db/Important Price Datasets/new_exit_income_final.dta", replace
}

use "$db/Important Price Datasets/new_exit_income_final.dta", clear
keep ssnp ssep income_quintile year
duplicates drop
br
tsset income_q year
gen ssep_final=L.ssep

drop if year==2004
order ssnp ssep_final
drop ssep
gen feenstra=(1-ssnp)/(1-ssep)

collapse (mean) ssnp ssep feenstra, by(income_quintile)
replace ssnp=ssnp*100
replace ssep=ssep*100

scatter ssnp income_quintile, c(l) || scatter ssep income_quintile, ///
c(l) ms(T) graphregion(color(white)) ylabel(0(2)11 ,angle(0)) xtitle("Household Income Quintile") ///
ytitle("Average Spending Share on New" "and Exiting Products, 2004-2015, %") ///
legend(pos(12) ring(0) region(lwidth(none)) label(1 "New") label(2 "Exiting")) 
graph export "$resultspath/Figure2PanelAa.pdf", as(pdf) replace
graph save "$resultspath/Figure2PanelAa.gph", replace

** Panel A(b) **

* Compute average consumer income by module during the sample
use "$db/Important Price Datasets/price_index_household_income_2004_2004_modules.dta", clear
foreach i of numlist 2005(1)2015 {
append using "$db/Important Price Datasets/price_index_household_income_`i'_2004_modules.dta"
}
bysort product_module_code: egen double totalspending_M = sum(projection_factor*total_spending)

drop if missing(household_income)
gen household_income_v2=2500 if household_income==3
replace household_income_v2=6500 if household_income==4
replace household_income_v2=9000   if household_income==6
replace household_income_v2=11000  if household_income==8
replace household_income_v2=13500  if household_income==10
replace household_income_v2=17500  if household_income==11
replace household_income_v2=22500  if household_income==13
replace household_income_v2=27500  if household_income==15
replace household_income_v2=32500  if household_income==16
replace household_income_v2=37500  if household_income==17
replace household_income_v2=42500  if household_income==18
replace household_income_v2=47500  if household_income==19
replace household_income_v2=55000  if household_income==21
replace household_income_v2=65000  if household_income==23
replace household_income_v2=85000  if household_income==26
replace household_income_v2=150000 if household_income==27
replace household_income_v2=150000 if household_income>27
bysort product_module_code: egen double temp = sum(projection_factor*total_spending*household_income_v2)

gen double consumer_income_salesw=temp/totalspending_M 
keep product_module_code totalspending_M consumer_income_salesw
duplicates drop
save "$db/Important Price Datasets/consumerincome_modules.dta", replace

* now get spending shares on new goods (from full population) in the module during the sample
use "$db/Important Price Datasets/new_exit_M_2004.dta", clear
foreach i of numlist 2005(1)2015 {
append using "$db/Important Price Datasets/new_exit_M_`i'.dta"
}
tsset product_module year
gen ssep_final=L.ssep
drop if year==2004
order ssnp ssep_final
drop ssepM
gen feenstra=(1-ssnp)/(1-ssep)
collapse (mean) ssnp ssep feenstra total_spending_M, by(product_module_code product_group_code)
merge 1:1 product_module_code using "$db/Important Price Datasets/consumerincome_modules.dta"

* exclude outliers
sum consumer_income_salesw, d
gen p5=r(p5)
gen p95=r(p95)
replace ssnp=ssnp*100

reg ssnp consumer_income_salesw if consumer_income_salesw>p5 & consumer_income_salesw<p95 [aw=log(total_spending_M)], cluster(product_group_code)
matrix b=e(b) 
local b: di %6.5f `=b[1,1]'
matrix var=e(V)
matrix sd = sqrt(var[1,1])
local sd: di %6.5f `=sd[1,1]'
display "`b'"
display "`sd'"

sum ssnpM, d
gen ssnpM_outlier=ssnp>=r(p99)

binscatter ssnpM consumer_income_salesw if  consumer_income_salesw>p5 & consumer_income_salesw<p95 & ssnpM_outlier==0 [aw=log(total_spending_M)], reportreg ///
xtitle("Sales-Weighted Consumer Income in Product Module, 2004-2015, $") ytitle("Average Spending Share on New" ///
"Products, 2004-2015, %") ylabel(,angle(0)) note("Coeff. `b' ***" "s.e. `sd'", span ring(0) pos(12) size(med) box bfc(white)) nq(100)
graph export "$resultspath/Figure2PanelAb.pdf", as(pdf) replace
graph save "$resultspath/Figure2PanelAb.gph", replace

